 
  

 






<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> 
<html>

<!-- Mirrored from www.javapractices.com/topic/TopicAction.do;jsessionid=4FCCB481C702D708A7360133D128E359?Id=147 by HTTrack Website Copier/3.x [XR&CO'2010], Sun, 12 Jun 2011 17:27:36 GMT -->
<!-- Added by HTTrack --><meta http-equiv="content-type" content="text/html;charset=UTF-8"><!-- /Added by HTTrack -->
<head>
 <title>
  Java Practices -> Fetch web page and header
 </title>
 <link rel="stylesheet" type="text/css" href="../stylesheet8.css" media="all">
 
 <link rel="shortcut icon" href='../images/favicon.ico' type="image/vnd.microsoft.icon">
 <meta name="description" content="Concise presentations of java programming practices, tasks, and conventions, amply illustrated with syntax highlighted code examples.">
 
 <meta name='keywords' content='URL,URLConnection,header,web,java,java programming,java practices,java idiom,java style,java design patterns,java coding conventions,'>
 
 
</head>
 
<body>


<div class='menu-bar'>
 
  <a href='../home/HomeAction.html' title='Table of Contents'>Home</a> |
  <a href='../vote/VoteSummaryAction-2.html' title='View Poll Results'>Poll</a> |
   
  <A href='../feedback/FeedbackAction451f-2.html?Operation=Show' title='Send Your Feedback'>Wiki</a> |
  <b><a href='../source/SourceAction-2.html' title='Grab Source Code'>Source Code</a></b><IMG class='no-margin' SRC="../images/goldstar.gif" ALT=""> |

  <a href='http://www.web4j.com/Java_Web_Application_Framework_Overview.jsp?From=1' title='Free Download - Java Web Application Framework'><b>WEB4J</b></a> |
  
  <a href='http://www.date4j.net/' title='Replacement for java.util.Date'><b>DATE4J</b></a> |

   <a href='../references/ReferencesAction-2.html' title='References'>Links</a>
   
  <form action='http://www.javapractices.com/search/SearchAction.do' method='get' class='search-form'>
   <input type='text' name='SearchTerms' value="" size=12 maxlength=50 class='search'>
   <input type='submit' value="Search">
  </form>
 
</div>

<P>



  

 






<p class="display-messages">

 

 

</p>


<div class="main-layout">
 
   

 




<div class='page-title'>Fetch web page and header</div>

<div class='main-body'>
 
<br>Here is an example of fetching the HTML content of a web page as simple
text.

<p>This could be used, for example, to fetch stock prices or the weather
forecast from the web - the raw HTML is first fetched, then the desired
content is extracted and presented in some customized manner.
<br>
<PRE>

<span class='keyword'>import</span> java.io.*;
<span class='keyword'>import</span> java.net.*;
<span class='keyword'>import</span> java.util.Scanner;

<span class='comment'>/** Fetches the HTML content of a web page (or HTTP header) as a String. */</span>
<span class='keyword'>public</span> <span class='keyword'>final</span> <span class='keyword'>class</span> WebPageFetcher {

  <span class='comment'>/**
  * Demo harness.
  * 
  * &lt;ul&gt;
  * &lt;li&gt;aArgs[0] : an HTTP URL
  * &lt;li&gt;aArgs[1] : (header | content)
  * &lt;/ul&gt;
  */</span>
  <span class='keyword'>public</span> <span class='keyword'>static</span> <span class='keyword'>void</span> main(String... aArgs) <span class='keyword'>throws</span> MalformedURLException {
    String url = aArgs[<span class='literal'>0</span>];
    String option = aArgs[<span class='literal'>1</span>];
    WebPageFetcher fetcher = <span class='keyword'>new</span>  WebPageFetcher(url);
    <span class='keyword'>if</span> ( HEADER.equalsIgnoreCase(option) ) {
      log( fetcher.getPageHeader() );
    }
    <span class='keyword'>else</span> <span class='keyword'>if</span> ( CONTENT.equalsIgnoreCase(option) ) {
      log( fetcher.getPageContent() );
    }
    <span class='keyword'>else</span> {
      log(<span class='literal'>"Unknown option."</span>);
    }
  }

  <span class='keyword'>public</span> WebPageFetcher( URL aURL ){
    <span class='keyword'>if</span> ( ! HTTP.equals(aURL.getProtocol())  ) {
      <span class='keyword'>throw</span> <span class='keyword'>new</span> IllegalArgumentException(<span class='literal'>"URL is not for HTTP Protocol: "</span> + aURL);
    }
    fURL = aURL;
  }

  <span class='keyword'>public</span> WebPageFetcher( String aUrlName ) <span class='keyword'>throws</span> MalformedURLException {
    <span class='keyword'>this</span> ( <span class='keyword'>new</span> URL(aUrlName) );
  }

  <span class='comment'>/** Fetch the HTML content of the page as simple text.   */</span>
  <span class='keyword'>public</span> String getPageContent() {
    String result = <span class='keyword'>null</span>;
    URLConnection connection = <span class='keyword'>null</span>;
    <span class='keyword'>try</span> {
      connection =  fURL.openConnection();
      Scanner scanner = <span class='keyword'>new</span> Scanner(connection.getInputStream());
      scanner.useDelimiter(END_OF_INPUT);
      result = scanner.next();
    }
    <span class='keyword'>catch</span> ( IOException ex ) {
      log(<span class='literal'>"Cannot open connection to "</span> + fURL.toString());
    }
    <span class='keyword'>return</span> result;
  }

  <span class='comment'>/** Fetch HTML headers as simple text.  */</span>
  <span class='keyword'>public</span> String getPageHeader(){
    StringBuilder result = <span class='keyword'>new</span> StringBuilder();

    URLConnection connection = <span class='keyword'>null</span>;
    <span class='keyword'>try</span> {
      connection = fURL.openConnection();
    }
    <span class='keyword'>catch</span> (IOException ex) {
      log(<span class='literal'>"Cannot open connection to URL: "</span> + fURL);
    }

    <span class='comment'>//not all headers come in key-value pairs - sometimes the key is
</span>    <span class='comment'>//null or an empty String
</span>    <span class='keyword'>int</span> headerIdx = <span class='literal'>0</span>;
    String headerKey = <span class='keyword'>null</span>;
    String headerValue = <span class='keyword'>null</span>;
    <span class='keyword'>while</span> ( (headerValue = connection.getHeaderField(headerIdx)) != <span class='keyword'>null</span> ) {
      headerKey = connection.getHeaderFieldKey(headerIdx);
      <span class='keyword'>if</span> ( headerKey != <span class='keyword'>null</span> &amp;&amp; headerKey.length()&gt;<span class='literal'>0</span> ) {
        result.append( headerKey );
        result.append(<span class='literal'>" : "</span>);
      }
      result.append( headerValue );
      result.append(NEWLINE);
      headerIdx++;
    }
    <span class='keyword'>return</span> result.toString();
  }

  <span class='comment'>// PRIVATE //
</span>  <span class='keyword'>private</span> URL fURL;
  
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>final</span> String HTTP = <span class='literal'>"http"</span>;
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>final</span> String HEADER = <span class='literal'>"header"</span>;
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>final</span> String CONTENT = <span class='literal'>"content"</span>;
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>final</span> String END_OF_INPUT = <span class='literal'>"&#092;&#092;Z"</span>;
  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>final</span> String NEWLINE = System.getProperty(<span class='literal'>"line.separator"</span>);

  <span class='keyword'>private</span> <span class='keyword'>static</span> <span class='keyword'>void</span> log(Object aObject){
    System.out.println(aObject);
  }
} 
</PRE>
<br>
<br>An example run, fetching an HTTP header from google.com :
<PRE>
>java -cp . WebPageFetcher http://www.google.com/ header
HTTP/1.1 200 OK
Cache-Control : private
Content-Type : text/html; charset=ISO-8859-1
Server : GWS/2.1
Transfer-Encoding : chunked
Date : Wed, 29 Aug 2007 13:21:40 GMT
</PRE>
<p>
<br>

</div>





<div class='topic-section'>Would you use this technique?</div>
<div class='main-body'>
  
  <form action="http://www.javapractices.com/vote/AddVoteAction.do" method='post'>
    Yes<input type='radio' name='Choice' value='Y' >
    &nbsp;&nbsp;No<input type='radio' name='Choice' value='N'>
    &nbsp;&nbsp;Undecided<input type='radio' name='Choice' value="?" >
    &nbsp;&nbsp;<input type=submit value="Vote" >
    <input type='hidden' name='Operation' value='Apply'>
    <input type='hidden' name='TopicId' value='147'>
  </form>
</div>

<div style='height:10.0em;'></div>

 
 
</div>

  

 





<div align='center' class='legalese'>  
&copy; 2011 Hirondelle Systems |
<a href='../source/SourceAction-2.html'><b>Source Code</b></a><IMG class='no-margin' SRC="../images/goldstar.gif" ALT=""> |
<a href="mailto:webmaster@javapractices.com">Contact</a> |
<a href="http://creativecommons.org/licenses/by-nc-sa/1.0/">License</a> |
<a href='../apps/cjp.rss'>RSS</a>
<!-- ukey="2AC36CD2" -->
<!-- ckey="16DF3D87" -->
<br>

 Individual code snippets can be used under this <a href='../LICENSE.txt'>BSD license</a> - Last updated on June 6, 2010.<br>
 Over 150,000 unique IPs last month - <span title='Java Practices 2.6.5, Mon May 16 00:00:00 EDT 2011'>Built with</span> <a href='http://www.web4j.com/'>WEB4J</a>.<br>
 - In Memoriam : Bill Dirani -
</div>

<script src="../../www.google-analytics.com/urchin.js" type="text/javascript">
</script>
<script type="text/javascript">
_uacct = "UA-2633428-1";
urchinTracker();
</script>



</body>

<!-- Mirrored from www.javapractices.com/topic/TopicAction.do;jsessionid=4FCCB481C702D708A7360133D128E359?Id=147 by HTTrack Website Copier/3.x [XR&CO'2010], Sun, 12 Jun 2011 17:27:36 GMT -->
<!-- Added by HTTrack --><meta http-equiv="content-type" content="text/html;charset=UTF-8"><!-- /Added by HTTrack -->
</html>
